package com.lheia.process.zhejiang.quzhou


import com.lheia.jcb.spider.IpeSpiderData
import com.lheia.process.BaseProcess
import com.lheia.tool.ToolHttpConnect
import com.lheia.tool.ToolString
import com.lheia.tool.ToolUserAgents
import com.lheia.util.PageFieldConstants
import com.lheia.util.RequestExtraConstants
import com.lheia.util.ToolUrl
import org.jsoup.nodes.DataNode
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Request
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.utils.HttpConstant


class SpiderSZFYQProcess implements BaseProcess {
    private Site site = Site.me().setCycleRetryTimes(3).setUserAgent(ToolUserAgents.userAgentRandom())
    private String province = "浙江省"
    private String city = "衢州市"
    private String county = ""
    private String approveDept = "市政府"


    void setIfTest(boolean ifTest) {
        this.ifTest = ifTest
    }
    protected boolean ifTest = true
    /**
     * process the page, extract urls to fetch, extract the data and store
     *
     * @param page page
     */
    @Override
    /**
     * 主要思路使用xpath解析列表也对象添加到请求队列中
     *
     */
    void process(Page page) {
        def methodName = page.getRequest().getExtra(RequestExtraConstants.CALLBACK)
        if (methodName) {
            this."$methodName"(page)
        }
//        processTargetResquest(page)
    }

    @Override
    def getBeginRequest(String url) {
        List<Request> list = new ArrayList<>()

        Request request = new Request(url)
        /**
         * 使用无头浏览器
         */
        request.putExtra(RequestExtraConstants.PHANTOM_JS, true)
        /**
         * 不计入去重
         */
        request.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
        /**
         * 处理回调
         */
        request.putExtra(RequestExtraConstants.CALLBACK, "processList")
        list << request

        return list
    }

    /***
     * 列表处理
     * @param page
     */
    void processList(Page page) {
        def doc = page.getHtml().getDocument().select("div#4717086")
        /**列表node查找***/
//        def trList = page.getHtml().getDocument().select("table.xzcf_lb").select("tr")
//        trList.each {
//            def listNode = it.getElementsByTag("a").get(0)
//            def detailUrl = listNode.attr("href")
//            def title = listNode.attr("title")
//
//            /***
//             * 过滤title是否抓取抓取之后
//             */
//            def abs = ToolUrl.getAbsoluteURL(page.getUrl().toString(), detailUrl)
//            def detailRequest = new Request(abs)
//            println String.join(",", [title, abs])
//            detailRequest.putExtra(RequestExtraConstants.CALLBACK, "processPagingCF")
//            page.addTargetRequest(detailRequest)
//        }
//        page.getResultItems().setSkip(true)
//        processPaging(page)
    }
    /**
     * 分页处理
     * @param page
     */
    void processPaging(Page page) {
        def doc = page.getHtml().getDocument()
        /***
         * 获取总页数
         */
        def pageTotalNum = ToolString.matchList(doc.select("li:contains(条记录)").text(), "\\d+")?.get(0)
        pageTotalNum = pageTotalNum ? Integer.valueOf(pageTotalNum) : 0
        def pevReq = page.getRequest()
        def queryData = pevReq.getExtra(RequestExtraConstants.PREV_QUERY_DATA)
        if (queryData) {
            queryData.pageNo = queryData.pageNo + 1
        } else {
            queryData = this.pageQueryData
        }
        /**
         * 若小于等于总页数则翻页
         */
        if (queryData.pageNo <= pageTotalNum) {
            def url = pevReq.getUrl()
            url = url.split("\\?")[0]
            url = ToolHttpConnect.appendUrlParam(url, queryData)
            def pageRequest = new Request(url)
            pageRequest.setMethod(HttpConstant.Method.GET)
            pageRequest.putExtra(RequestExtraConstants.PREV_QUERY_DATA, queryData)
            /***
             * 分页请求不需要进行去重
             */
            pageRequest.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            /**
             * 是否使用phantomJS
             */
            pageRequest.putExtra(RequestExtraConstants.PHANTOM_JS, page.getRequest().getExtra(RequestExtraConstants.PHANTOM_JS))
            /**
             * 处理回调
             */
            pageRequest.putExtra(RequestExtraConstants.CALLBACK, "processList")
            println("下一页," + pageRequest)
            page.addTargetRequest(pageRequest)
        }
    }

    void processPagingCF(Page page) {
        def doc = page.getHtml().getDocument()
        def ipeSpiderData = new IpeSpiderData()
        ipeSpiderData.companyName = doc.select("td.xzcf_xx").get(1).text()
        ipeSpiderData.messageSource = doc.select("td.xzcf_xx").get(0).text()
        ipeSpiderData.pubOrg = doc.select("td.xzcf_xx").get(2).text()
        ipeSpiderData.province = this.province
        ipeSpiderData.city = this.city
        ipeSpiderData.county = this.county
        ipeSpiderData.illegalDetail = doc.select("td.xzcf_jds").text()
        ipeSpiderData.status = "处罚"
        ipeSpiderData.timeOccurrence = com.lheia.tool.ToolDateTime.extractDate(doc.select("td.xzcf_xx").get(3).text())
        page.putField(PageFieldConstants.RESULT_DATA, ipeSpiderData)
    }


    /**
     * 如果为测试环境或者开发环境不计入redis去重只使用redisList队列
     * @param page
     */
    void processTargetResquest(Page page) {
        if (ifTest) {
            println(page)
            page.getTargetRequests().each {
                it.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            }
        }
    }


/**
 * get the site settings
 *
 * @return site* @see Site
 */
    @Override
    Site getSite() {
        return site
    }
}
